const { Document } = require('@langchain/core/documents');
const BaseRagService = require('./baseRag');
const path = require('path');
const {ragUrlsConfig} = require('./data/webRagConfig');
const VectorStoreService = require('../vectorStore');
const axios = require('axios');
const https = require('https');
const cheerio = require('cheerio');

// 忽略SSL证书错误
axios.defaults.httpsAgent = new https.Agent({  
  rejectUnauthorized: false // 设置为 false 以忽略 SSL 证书错误
});

class WebRagService extends BaseRagService {
  static COLLECTION_NAME = 'webRag';
  static RAW_DATA_DIR = path.join(process.cwd(), 'core', 'rag', 'data', 'urls.json');

  // 初始化数据
  static async initializeData() {
    try {
      //判断向量存储的数据是否存在 不存在则处理文档生成
      if(!VectorStoreService.vectorStoreCollectionExists(this.COLLECTION_NAME)){
        // 初始化向量存储
        await VectorStoreService.initCollection(this.COLLECTION_NAME);
        // 处理所有配置的URL
        for (const urlItem of ragUrlsConfig) {
          try {
            await this.loadFromUrl(urlItem.url);
            console.log(`成功加载URL内容: ${urlItem.title}`);
          } catch (error) {
            console.error(`加载URL失败 ${urlItem.title}:`, error.message);
          }
        }
      }else {
        // 初始化向量存储
        await VectorStoreService.initCollection(this.COLLECTION_NAME);
        console.log('Web型RAG集合数据已存在,将会使用原有的集合！');
      }
    } catch (error) {
      console.error('初始化webRag数据失败:', error.message);
      throw error;
    }
  }

  // 从URL加载网页内容
  static async loadFromUrl(url) {
    try {
      if (!url) {
        throw new Error('URL不能为空');
      }

      const response = await axios.get(url);
      const $ = cheerio.load(response.data);
      
      // 移除脚本和样式标签
      $('script').remove();
      $('style').remove();
      
      // 提取并清理文本内容
      const content = $('body').text()
        .replace(/\s+/g, ' ')
        .trim();
      
      if (!content) {
        throw new Error('提取的网页内容为空');
      }

      const doc = new Document({
        pageContent: content,
        metadata: {
          source_url: url,
          content_type: 'html',
          created_at: new Date().toISOString()
        }
      });

      return this.processContent([doc], this.COLLECTION_NAME, { source_url: url });
    } catch (error) {
      console.error('从URL加载内容失败:', error.message);
      throw error;
    }
  }

  // 从HTML内容加载
  static async loadFromHtml(html, metadata = {}) {
    try {
      if (!html) {
        throw new Error('HTML内容不能为空');
      }

      const $ = cheerio.load(html);
      
      // 移除脚本和样式标签
      $('script').remove();
      $('style').remove();
      
      // 提取并清理文本内容
      const content = $('body').text()
        .replace(/\s+/g, ' ')
        .trim();
      
      if (!content) {
        throw new Error('提取的HTML内容为空');
      }

      const doc = new Document({
        pageContent: content,
        metadata: {
          ...metadata,
          content_type: 'html',
          created_at: new Date().toISOString()
        }
      });

      return this.processContent([doc], this.COLLECTION_NAME, metadata);
    } catch (error) {
      console.error('从HTML加载内容失败:', error.message);
      throw error;
    }
  }
}

module.exports = WebRagService;